Advanced Lane Finding Project


Import Packages

In [1]:
#importing some useful packages
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import cv2
import pickle
import glob
import PIL
import shutil
from io import BytesIO
from collections import deque
from moviepy.editor import VideoFileClip
from IPython.display import HTML
from IPython.display import Image, display
%matplotlib inline

Ideas for Advanced Lane Detection Pipeline

The goals / steps of this project are the following:

  • Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.
  • Apply a distortion correction to raw images.
  • Use color transforms, gradients, etc., to create a thresholded binary image.
  • Apply a perspective transform to rectify binary image ("birds-eye view").
  • Detect lane pixels and fit to find the lane boundary.
  • Determine the curvature of the lane and vehicle position with respect to center.
  • Warp the detected lane boundaries back onto the original image.
  • Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.

Helper Functions

In [2]:
def find_camera_calibration(images, cal_width = 9, cal_height = 6):
    # Prepare object points, like (0,0,0), (1,0,0), (2,0,0) , ... , (6,5,0)
    objp = np.zeros((cal_width * cal_height,3), np.float32)
    objp[:,:2] = np.mgrid[0:cal_width, 0:cal_height].T.reshape(-1,2)

    # Arrays to store object points and image points from all the images.
    objpoints = []  # 3d points in real world space
    imgpoints = []  # 2d points in image plane

    # Step through the list and search for chessboard corners
    for idx, fname in enumerate(images):
        img = cv2.imread(fname)
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
        
        # Find the chessboard corners
        ret, corners = cv2.findChessboardCorners(gray, (cal_width, cal_height), None)

        # If found, add object points, image points
        if ret == True:
            print('working on ', fname)
            objpoints.append(objp)
            imgpoints.append(corners)

            # Draw and display the corners
            cv2.drawChessboardCorners(img, (cal_width, cal_height), corners, ret)
            write_name = './camera_cal/corners_found'+str(idx+1)+'.jpg'
            cv2.imwrite(write_name, img)

    # load image for reference
    img = cv2.imread('./camera_cal/calibration1.jpg')
    img_size = (img.shape[1], img.shape[0])

    # Do camera calibration given object points and image points
    ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, img_size, None, None)

    # Save the camera calibration result for later use (we won't worry about rvecs / tvecs)
    dist_pickle = {}
    dist_pickle["mtx"] = mtx
    dist_pickle["dist"] = dist
    pickle.dump(dist_pickle, open("./camera_cal/calibration_pickle.p", "wb"))

# Define a function that applies Sobel x or y
def abs_sobel_thresh(img, orient='x', sobel_kernel=3, thresh=(0, 255)):
    # Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    if orient == 'x':
        abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel))
    if orient == 'y':
        abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel))
    scaled_sobel = np.uint8(255*abs_sobel/np.max(abs_sobel))
    binary_output = np.zeros_like(scaled_sobel)
    # Apply threshold
    binary_output[(scaled_sobel >= thresh[0]) & (scaled_sobel <= thresh[1])] = 1
    return binary_output

def color_threshold(img, sthresh=(0, 255), vthresh=(0, 255)):
    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    s_channel = hls[:,:,2]
    s_binary = np.zeros_like(s_channel)
    s_binary[(s_channel >= sthresh[0]) & (s_channel <= sthresh[1])] = 1

    hsv = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
    v_channel = hsv[:,:,2]
    v_binary = np.zeros_like(v_channel)
    v_binary[(v_channel >= vthresh[0]) & (v_channel <= vthresh[1])] = 1

    output = np.zeros_like(s_channel)
    output[(s_binary == 1) & (v_binary == 1)] = 1
    return output, s_binary, v_binary

def preprocess_image(img, mtx, dist):
    # Image already in RGB format

    # Undistort the image
    img = cv2.undistort(img, mtx, dist, None, mtx)

    # Calculate image size
    img_size = (img.shape[1], img.shape[0])

    # process image and generate binary pixel of interests
    preprocessImage = np.zeros_like(img[:,:,0])
    gradx = abs_sobel_thresh(img, orient='x', sobel_kernel=3, thresh=(12, 100))
    grady = abs_sobel_thresh(img, orient='y', sobel_kernel=3, thresh=(25, 100))
    c_binary, s_binary, v_binary = color_threshold(img, sthresh=(100, 255), vthresh=(150, 255))
    preprocessImage[((gradx == 1) & (grady == 1) | (c_binary == 1))] = 255
    return preprocessImage, img, img_size

def find_src_dst_roi_vertices(bot_width, mid_width, height_pct, bottom_trim, img_size):
    # Define properties transformation area
    src = np.float32([[img_size[0]*(.5-mid_width/2), img_size[1]*height_pct], [img_size[0]*(.5+mid_width/2), img_size[1]*height_pct], [img_size[0]*(.5+bot_width/2), img_size[1]*bottom_trim],[img_size[0]*(.5-bot_width/2), img_size[1]*bottom_trim]])
    offset = img_size[0]*.25
    dst = np.float32([[offset, 0], [img_size[0]-offset, 0], [img_size[0]-offset, img_size[1]], [offset, img_size[1]]])

    roi_left_padding = img_size[0]*.20
    roi_right_padding = img_size[0]*.15
    roi_vertices =  np.array([
                        [roi_left_padding, 0],
                        [img_size[0] - roi_right_padding, 0],
                        [img_size[0] - roi_right_padding, img_size[1]], 
                        [roi_left_padding, img_size[1]]]
                        , dtype=np.int32)
    return src, dst, roi_vertices

def find_perspective_transform(src, dst, preprocessImage, img_size):
    # Perform the transform
    M = cv2.getPerspectiveTransform(src, dst)
    Minv = cv2.getPerspectiveTransform(dst, src)
    warped = cv2.warpPerspective(preprocessImage, M, img_size, flags=cv2.INTER_LINEAR)
    return warped, Minv, M

def get_roi(img, vertices):
    # Applies an image mask
    # defining a blank mask to start with
    mask = np.zeros_like(img)
    # defining a 3 channel or 1 channel color to fill the mask with depending on the input image
    if len(img.shape) > 2:
        channel_count = img.shape[2]
        ignore_mask_color = (255,) * channel_count
    else:
        ignore_mask_color = 255
        
    # filling pixels inside the polygon defined by "vertices" with the fill color    
    cv2.fillPoly(mask, np.array([vertices]), ignore_mask_color)
    
    # returning the image only where mask pixels are nonzero
    masked_image = cv2.bitwise_and(img, mask)
    return masked_image

def find_window_centroids(warped, window_width, window_height, margin, minpix, recent_centers, smooth_factor):
    window = np.ones(window_width)   # Create our window template that we will use for convolutions
    window_centroids = []            # Store the (left, right) window centroid positions per level

    # First find the two starting positions for the left and right lane by using np.sum to get the vertical image slice
    # and then np.convolve the vertical image slice with the window template

    # Sum quarter bottom of image to get slice, could use a different ratio
    l_sum = np.sum(warped[int(3*warped.shape[0]/4):, :int(warped.shape[1]/2)], axis=0)
    l_center = np.argmax(np.convolve(window, l_sum))-window_width/2
    r_sum = np.sum(warped[int(3*warped.shape[0]/4):, int(warped.shape[1]/2):], axis=0)
    r_center = np.argmax(np.convolve(window, r_sum))-window_width/2+int(warped.shape[1]/2)

    # Add what we found for the first layer
    window_centroids.append((l_center, r_center))

    # Go through each layer looking for max pixel locations
    for level in range(1, (int)(warped.shape[0]/window_height)):
        # convolve the window into the vertical slice of the image
        image_layer = np.sum(warped[int(warped.shape[0]-(level+1)*window_height):int(warped.shape[0]-level*window_height),:], axis=0)
        conv_signal = np.convolve(window, image_layer)
        # Find the best left centroid by using past left center as a reference
        # Use window_width/2 as offset because convolution signal reference is at right side of the window, not center of window
        offset = window_width/2        
        
        # Find the left centroid by using past right center as a reference
        l_min_index = int(max(l_center+offset-margin, 0))
        l_max_index = int(min(l_center+offset+margin, warped.shape[1]))

        l_signal = np.max(conv_signal[l_min_index:l_max_index])
        if l_signal > minpix:    # If total pixels > minpix pixels, recenter next window on their highest convolution-value position else center value remains same as the last value
            l_center = np.argmax(conv_signal[l_min_index:l_max_index])+l_min_index-offset

        # Find the best right centroid by using past right center as a reference
        r_min_index = int(max(r_center+offset-margin, 0))
        r_max_index = int(min(r_center+offset+margin, warped.shape[1]))

        r_signal = np.max(conv_signal[r_min_index:r_max_index])
        if r_signal > minpix:        # If total pixels > minpix pixels, recenter next window on their highest convolution-value position
            r_center = np.argmax(conv_signal[r_min_index:r_max_index])+r_min_index-offset

        # Add what we found for that layer
        window_centroids.append((l_center, r_center))

    recent_centers.append(window_centroids)

    # return averaged values of the line centers, helps to keep the markers from jumping around too much
    return np.average(np.array(recent_centers)[-smooth_factor:], axis=0), recent_centers

def window_mask(width, height, img_ref, center, level):
    output = np.zeros_like(img_ref)
    output[int(img_ref.shape[0]-(level+1)*height):int(img_ref.shape[0]-level*height),max(0,int(center-width/2)):min(int(center+width/2),img_ref.shape[1])] = 1
    return output

def find_lane_mask_pixels(window_width, window_height, warped_cropped, window_centroids):
    # points used to find the left and right lanes based on window centroids
    leftx = []
    rightx = []

    # Points used to draw all the left and rigth windows
    l_points = np.zeros_like(warped_cropped)
    r_points = np.zeros_like(warped_cropped)

    # If we found any window centers
    if len(window_centroids) > 0:
        # Go through each level and draw the Windows
        for level in range(0, len(window_centroids)):
            # add center value found in frame to the list of lane points per left, right
            leftx.append(window_centroids[level][0])
            rightx.append(window_centroids[level][1])        
            # Window_mask is a function to draw window areas
            l_mask = window_mask(window_width, window_height, warped_cropped, window_centroids[level][0], level)
            r_mask = window_mask(window_width, window_height, warped_cropped, window_centroids[level][1], level)
            # Add graphic points from window mask here to total pixels found
            l_points[(l_points == 255) | ((l_mask == 1))] = 255
            r_points[(r_points == 255) | ((r_mask == 1))] = 255    
    return leftx, rightx, l_points, r_points

def draw_results_left_right_window(l_points, r_points, warped_cropped, window_centroids):
    # If we found any window centers
    if len(window_centroids) > 0:
        template = np.array(l_points + r_points, np.uint8)    # add both left and rigth window pixels together
        zero_channel = np.zeros_like(template)  # create a zero color channel
        template = np.array(cv2.merge((zero_channel, template, zero_channel)), np.uint8)    # make window pixels green
        warpage = np.array(cv2.merge((warped_cropped, warped_cropped, warped_cropped)), np.uint8)   # making the original road pixels 3 color channels
        result_raw = cv2.addWeighted(warpage, 1, template, 0.5, 0.0)    # overlay the original road image with window results

    # If no window centers found, just display orginal road image
    else:
        result_raw = np.array(cv2.merge((warped_cropped,warped_cropped,warped_cropped)),np.uint8)
    return result_raw

def find_lane_boundaries(warped_cropped, window_height, leftx, rightx, window_width, l_points, r_points):

    yvals = range(0, warped_cropped.shape[0])
    res_yvals = np.arange(warped_cropped.shape[0]-(tracker.window_height/2), 0, -tracker.window_height)

    left_fit = np.polyfit(res_yvals, leftx, 2)
    right_fit = np.polyfit(res_yvals, rightx, 2)

    left_fitx = left_fit[0]*yvals*yvals + left_fit[1]*yvals + left_fit[2]
    left_fitx = np.array(left_fitx, np.int32)

    right_fitx = right_fit[0]*yvals*yvals + right_fit[1]*yvals + right_fit[2]
    right_fitx = np.array(right_fitx, np.int32)

    left_lane = np.array(list(zip(np.concatenate((left_fitx-tracker.window_width/2, left_fitx[::-1]+tracker.window_width/2), axis=0), np.concatenate((yvals, yvals[::-1]), axis=0))), np.int32)
    right_lane = np.array(list(zip(np.concatenate((right_fitx-tracker.window_width/2, right_fitx[::-1]+tracker.window_width/2), axis=0), np.concatenate((yvals, yvals[::-1]), axis=0))), np.int32)
    inner_lane = np.array(list(zip(np.concatenate((left_fitx+tracker.window_width/2, right_fitx[::-1]-tracker.window_width/2), axis=0), np.concatenate((yvals, yvals[::-1]), axis=0))), np.int32)
    return left_lane, right_lane, inner_lane, yvals, res_yvals, left_fitx, right_fitx

def draw_results_left_right_inner_lines(img, left_lane, right_lane, inner_lane, Minv, img_size):
    road = np.zeros_like(img)
    road_bkg = np.zeros_like(img)
    cv2.fillPoly(road, [left_lane], color=[255, 0, 0])
    cv2.fillPoly(road, [right_lane], color=[0, 0, 255])
    cv2.fillPoly(road, [inner_lane], color=[0, 255, 0])
    cv2.fillPoly(road_bkg, [left_lane], color=[255, 255, 255])
    cv2.fillPoly(road_bkg, [right_lane], color=[255, 255, 255])

    road_warped = cv2.warpPerspective(road, Minv, img_size, flags=cv2.INTER_LINEAR)
    road_warped_bkg = cv2.warpPerspective(road_bkg, Minv, img_size, flags=cv2.INTER_LINEAR)

    base = cv2.addWeighted(img, 1.0, road_warped_bkg, -1.0, 0.0)
    result = cv2.addWeighted(base, 1.0, road_warped, 1.0, 0.0)
    return result

def calculate_curvature_offset(res_yvals, yvals, leftx, rightx, ym_per_pix, xm_per_pix, left_fitx, right_fitx, warped):
    # calculate radius of left curvature
    curve_fit_cr_left = np.polyfit(np.array(res_yvals, np.float32)*ym_per_pix, np.array(leftx, np.float32)*xm_per_pix, 2)
    curverad_left = ((1 + (2*curve_fit_cr_left[0]*yvals[-1]*ym_per_pix + curve_fit_cr_left[1])**2)**1.5) / np.absolute(2*curve_fit_cr_left[0])

    # calculate radius of right curvature
    curve_fit_cr_right = np.polyfit(np.array(res_yvals, np.float32)*ym_per_pix, np.array(rightx, np.float32)*xm_per_pix, 2)
    curverad_right = ((1 + (2*curve_fit_cr_right[0]*yvals[-1]*ym_per_pix + curve_fit_cr_right[1])**2)**1.5) / np.absolute(2*curve_fit_cr_right[0])

    # calculate the offset of the car on the road
    camera_center = (left_fitx[-1] + right_fitx[-1])/2
    center_diff = (camera_center-warped.shape[1]/2)*xm_per_pix
    side_pos = 'left'
    if center_diff <= 0:
        side_pos = 'right'
    return curverad_left, curverad_right, center_diff, side_pos

def draw_curvature_offset(result, curverad_left, curverad_right, center_diff, side_pos):
    cv2.putText(result, 'Radius of left Curvature = '+ str(round(curverad_left, 3)) + '(m)', (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
    cv2.putText(result, 'Radius of right Curvature = '+ str(round(curverad_right, 3)) + '(m)', (50, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
    cv2.putText(result, 'Vehicle is ' + str(abs(round(center_diff, 3))) + 'm ' + side_pos + ' of center', (50, 150), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
    return result

def display_img_from_array(img_array):
    image_from_array = PIL.Image.fromarray(img_array)
    bio = BytesIO()
    image_from_array.save(bio, format='png')
    display(Image(bio.getvalue(), format='png'))

Helper Class

In [3]:
class Tracker():
    
    def __init__(self, Mycalib_width = 9, Mycalib_height = 6, Mywindow_width = 25, Mywindow_height = 80, Mymargin = 50, Myminpix = 50000, My_ym = 10/720, My_xm = 4/384, Mysmooth_factor = 15, Mybot_width = .76, Mymid_width = .08, Myheight_pct = .62, Mybottom_trim = .935):
        # the width of chessboard image used for camera calibration
        self.cal_width = Mycalib_width

        # the heigth of chessboard image used for camera calibration
        self.cal_height = Mycalib_height
        
        # the window pixel width of the center values, used to count pixels inside the center windows to determine curve values
        self.window_width = Mywindow_width

        # the window pixel height of the center values, used to count pixels inside center windows to determine curve values breaks the image into vertical levels
        self.window_height = Mywindow_height

        # The pixel distance in both directions to slide (left_window + right_window) template for searching
        self.margin = Mymargin

        # The pixel count needed at minimum to update window center 
        self.minpix = Myminpix

        # meters per pixel in vertical axis
        self.ym_per_pix = My_ym

        # meters per pixel in horizontal axis
        self.xm_per_pix = My_xm

        # to take average values based on previous frames
        self.smooth_factor = Mysmooth_factor

        # percent of bottom trapizoid height
        self.bot_width = Mybot_width

        # percent of middle trapizoid height
        self.mid_width = Mymid_width

        # percent for trapizoid height
        self.height_pct = Myheight_pct

        # percent from top to bottom to avoid car hood
        self.bottom_trim = Mybottom_trim

        # list that stores all the past (left, right) center set values used for smoothing the output
        self.recent_centers = deque(maxlen=100)             # memory from past 100 frames
        
        # total count of frames or images corresponding to data saved in memory
        self.total = len(self.recent_centers)

        # Read in the saved camera calibration data
        self.dist_pickle = None

        # Read in the saved calibration camera matrix
        self.mtx = None

        # Read in the saved calibration distortion coefficients
        self.dist = None

    def clear(self):
        """clear the tracker"""
        self.recent_centers.clear()
        self.total = 0

    def set_camera_calibration(self):
        """set camera calibration"""
        self.dist_pickle = pickle.load(open("./camera_cal/calibration_pickle.p", "rb"))
        self.mtx = self.dist_pickle["mtx"]
        self.dist = self.dist_pickle["dist"]

Set a tracker

To define constants and to store all the past (left, right) center set values used for smoothing the output

In [4]:
# Set up the overall class to do all the tracking
tracker = Tracker(Mycalib_width = 9, Mycalib_height = 6, Mywindow_width = 25, Mywindow_height = 80, Mymargin = 50, Myminpix = 50000, My_ym = 10/720, My_xm = 4/384, Mysmooth_factor = 15, Mybot_width = .76, Mymid_width = .08, Myheight_pct = .62, Mybottom_trim = .935)

Next, I will compute the camera calibration using chessboard images

In [5]:
# Make a list of calibration images
images = glob.glob('./camera_cal/calibration*.jpg')

# calibrate camera
find_camera_calibration(images, tracker.cal_width, tracker.cal_height)
working on  ./camera_cal/calibration14.jpg
working on  ./camera_cal/calibration16.jpg
working on  ./camera_cal/calibration7.jpg
working on  ./camera_cal/calibration6.jpg
working on  ./camera_cal/calibration19.jpg
working on  ./camera_cal/calibration2.jpg
working on  ./camera_cal/calibration15.jpg
working on  ./camera_cal/calibration20.jpg
working on  ./camera_cal/calibration10.jpg
working on  ./camera_cal/calibration8.jpg
working on  ./camera_cal/calibration12.jpg
working on  ./camera_cal/calibration13.jpg
working on  ./camera_cal/calibration11.jpg
working on  ./camera_cal/calibration17.jpg
working on  ./camera_cal/calibration3.jpg
working on  ./camera_cal/calibration9.jpg
working on  ./camera_cal/calibration18.jpg

Display chessboard image before calibration

In [6]:
# chessboard image before calibration
display(Image(filename='./camera_cal/calibration2.jpg'))
shutil.copy('./camera_cal/calibration2.jpg', './output_images/calibration2.jpg')
Out[6]:
'./output_images/calibration2.jpg'

Display chessboard image after corners found for camera calibration

In [7]:
# chessboard image after calibration
display(Image(filename='./camera_cal/corners_found6.jpg'))
shutil.copy('./camera_cal/corners_found6.jpg', './output_images/calibration2_corners_found6.jpg')
Out[7]:
'./output_images/calibration2_corners_found6.jpg'

Display chessboard image after undistortion

In [8]:
# chessboard before unditortion
test_image = './camera_cal/calibration2.jpg'

tracker.clear()
test_image = cv2.imread(test_image)
test_image = cv2.cvtColor(test_image, cv2.COLOR_BGR2RGB)

# set camera calibration
tracker.set_camera_calibration()

# process image and generate binary pixel of interests and undistorted image
preprocessImage, undistored_image, img_size = preprocess_image(test_image, tracker.mtx, tracker.dist)

display_img_from_array(undistored_image)
write_name = './output_images/calibration2_undistored_image.jpg'
cv2.imwrite(write_name, cv2.cvtColor(undistored_image, cv2.COLOR_BGR2RGB))
Out[8]:
True

Build an Advanced Lane Finding Pipeline

In [9]:
def process_image(img):
    # Image already in RGB format

    # process image and generate binary pixel of interests
    preprocessImage, undistored_image, img_size = preprocess_image(img, tracker.mtx, tracker.dist)

    # Define properties transformation area
    src, dst, roi_vertices = find_src_dst_roi_vertices(tracker.bot_width, tracker.mid_width, tracker.height_pct, tracker.bottom_trim, img_size)

    # find perspective transform
    warped, Minv, M = find_perspective_transform(src, dst, preprocessImage, img_size)

    # crop image
    warped_cropped = get_roi(warped, roi_vertices)

    # find window centroids
    window_centroids, tracker.recent_centers = find_window_centroids(warped, tracker.window_width, tracker.window_height, tracker.margin, tracker.minpix, tracker.recent_centers, tracker.smooth_factor)

    # find lane mask pixels
    leftx, rightx, l_points, r_points = find_lane_mask_pixels(tracker.window_width, tracker.window_height, warped_cropped, window_centroids)

    # draw the results with both left and right window
    result_raw = draw_results_left_right_window(l_points, r_points, warped_cropped, window_centroids)

    # fit the lane boundaries to the left, right center positions found
    left_lane, right_lane, inner_lane, yvals, res_yvals, left_fitx, right_fitx = find_lane_boundaries(warped_cropped, tracker.window_height, leftx, rightx, tracker.window_width, l_points, r_points)

    # draw the results with final left, right and inner lane lines
    result = draw_results_left_right_inner_lines(undistored_image, left_lane, right_lane, inner_lane, Minv, img_size)

    # calculate radius of left & right curvature and the offset of the car on the road
    curverad_left, curverad_right, center_diff, side_pos = calculate_curvature_offset(res_yvals, yvals, leftx, rightx, tracker.ym_per_pix, tracker.xm_per_pix, left_fitx, right_fitx, warped)

    # draw the text showing curvature and offset
    result = draw_curvature_offset(result, curverad_left, curverad_right, center_diff, side_pos)
    
    return result

Test on Images

In [10]:
# Set up the overall class to do all the tracking
tracker = Tracker(Mycalib_width = 9, Mycalib_height = 6, Mywindow_width = 25, Mywindow_height = 80, Mymargin = 50, Myminpix = 50000, My_ym = 10/720, My_xm = 4/384, Mysmooth_factor = 15, Mybot_width = .76, Mymid_width = .08, Myheight_pct = .62, Mybottom_trim = .935)

# set camera calibration
tracker.set_camera_calibration()

# Make a list of test images and save the results
images = glob.glob('./test_images/test*.jpg')
for idx, img in enumerate(images):
    img = cv2.imread(img)                       # Read in image
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # Convert BGR to RGB
    tracker.clear()                             # clear tracking data from previous frames
    result = process_image(img)
    write_name = './test_images/tracked'+str(idx+1)+'.jpg'
    cv2.imwrite(write_name, cv2.cvtColor(result, cv2.COLOR_BGR2RGB))

Display intermediate stages during the process of advanced lane-lines finding on a test image

In [11]:
# Display intermediate stages during the process of advanced lane-lines finding on a test image
img = './test_images/test1.jpg'
img = cv2.imread(img)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
tracker.clear()
tracker.set_camera_calibration()

# Original image in RGB format
display_img_from_array(img)
write_name = './output_images/test1_original.jpg'
cv2.imwrite(write_name, cv2.cvtColor(img, cv2.COLOR_BGR2RGB))

# Undistorted original image, and preprocessed binary image with binary pixel of interests generated
preprocessImage, undistored_image, img_size = preprocess_image(img, tracker.mtx, tracker.dist)
display_img_from_array(undistored_image)
write_name = './output_images/test1_undistorted.jpg'
cv2.imwrite(write_name, cv2.cvtColor(undistored_image, cv2.COLOR_BGR2RGB))
display_img_from_array(preprocessImage)
write_name = './output_images/test1_preprocessed.jpg'
cv2.imwrite(write_name, cv2.cvtColor(preprocessImage, cv2.COLOR_BGR2RGB))

# Define properties transformation area
src, dst, roi_vertices = find_src_dst_roi_vertices(tracker.bot_width, tracker.mid_width, tracker.height_pct, tracker.bottom_trim, img_size)

# find perspective transform
warped, Minv, M = find_perspective_transform(src, dst, preprocessImage, img_size)

# verify perespective transform
undistored_rgb_image = cv2.warpPerspective(img, M, img_size, flags=cv2.INTER_LINEAR)
display_img_from_array(undistored_rgb_image)
write_name = './output_images/test1_undistored_rgb_image.jpg'
cv2.imwrite(write_name, cv2.cvtColor(undistored_rgb_image, cv2.COLOR_BGR2RGB))

# cropped image within region of interest
warped_cropped = get_roi(warped, roi_vertices)
display_img_from_array(warped_cropped)
write_name = './output_images/test1_warped_cropped.jpg'
cv2.imwrite(write_name, cv2.cvtColor(warped_cropped, cv2.COLOR_BGR2RGB))

# find window centroids
window_centroids, tracker.recent_centers = find_window_centroids(warped, tracker.window_width, tracker.window_height, tracker.margin, tracker.minpix, tracker.recent_centers, tracker.smooth_factor)

# find lane mask pixels
leftx, rightx, l_points, r_points = find_lane_mask_pixels(tracker.window_width, tracker.window_height, warped_cropped, window_centroids)

# draw the results with both left and right window
result_raw = draw_results_left_right_window(l_points, r_points, warped_cropped, window_centroids)
display_img_from_array(result_raw)
write_name = './output_images/test1_result_raw.jpg'
cv2.imwrite(write_name, cv2.cvtColor(result_raw, cv2.COLOR_BGR2RGB))

# fit the lane boundaries to the left, right center positions found
left_lane, right_lane, inner_lane, yvals, res_yvals, left_fitx, right_fitx = find_lane_boundaries(warped_cropped, tracker.window_height, leftx, rightx, tracker.window_width, l_points, r_points)

# draw the results with final left, right and inner lane lines
result = draw_results_left_right_inner_lines(undistored_image, left_lane, right_lane, inner_lane, Minv, img_size)
display_img_from_array(result)
write_name = './output_images/test1_result.jpg'
cv2.imwrite(write_name, cv2.cvtColor(result, cv2.COLOR_BGR2RGB))

# calculate radius of left & right curvature and the offset of the car on the road
curverad_left, curverad_right, center_diff, side_pos = calculate_curvature_offset(res_yvals, yvals, leftx, rightx, tracker.ym_per_pix, tracker.xm_per_pix, left_fitx, right_fitx, warped)

# draw the text showing curvature and offset
result = draw_curvature_offset(result, curverad_left, curverad_right, center_diff, side_pos)
display_img_from_array(result)
write_name = './output_images/test1_result.jpg'
cv2.imwrite(write_name, cv2.cvtColor(result, cv2.COLOR_BGR2RGB))
Out[11]:
True
In [12]:
print("Source vertices (src)")
print(src.astype(int))
print("Destination vertices (dst)")
print(dst.astype(int))
print("Region of interest vertices (roi_vertices)")
print(roi_vertices.astype(int))
Source vertices (src)
[[ 588  446]
 [ 691  446]
 [1126  673]
 [ 153  673]]
Destination vertices (dst)
[[320   0]
 [960   0]
 [960 720]
 [320 720]]
Region of interest vertices (roi_vertices)
[[ 256    0]
 [1088    0]
 [1088  720]
 [ 256  720]]

Display the initial test images without lane-lines highlighted

In [13]:
# Make a list of test images and save the results
images = glob.glob('./test_images/test*.jpg')
for idx, img in enumerate(images):
    display(Image(filename='./test_images/test'+str(idx+1)+'.jpg'))

Display the final test images with lane-lines highlighted

In [14]:
# Make a list of test images and save the results
images = glob.glob('./test_images/test*.jpg')
for idx, img in enumerate(images):
    display(Image(filename='./test_images/tracked'+str(idx+1)+'.jpg'))

Test on Videos

In [15]:
# Set up the overall class to do all the tracking
tracker = Tracker(Mycalib_width = 9, Mycalib_height = 6, Mywindow_width = 25, Mywindow_height = 80, Mymargin = 50, Myminpix = 50000, My_ym = 10/720, My_xm = 4/384, Mysmooth_factor = 15, Mybot_width = .76, Mymid_width = .08, Myheight_pct = .62, Mybottom_trim = .935)

# set camera calibration
tracker.set_camera_calibration()

# Make a list of test images and save the results
Input_video = './project_video.mp4'
Output_video = './output1_tracked.mp4'

clip1 = VideoFileClip(Input_video)
tracker.clear()                                 # clear tracking data from previous frames
video_clip = clip1.fl_image(process_image)      # NOTE: this function expects color images!!
%time video_clip.write_videofile(Output_video, audio=False)
[MoviePy] >>>> Building video ./output1_tracked.mp4
[MoviePy] Writing video ./output1_tracked.mp4
100%|█████████▉| 1260/1261 [01:32<00:00, 13.17it/s]
[MoviePy] Done.
[MoviePy] >>>> Video ready: ./output1_tracked.mp4 

CPU times: user 4min 19s, sys: 2.46 s, total: 4min 22s
Wall time: 1min 32s

Play the video inline, or if you prefer find the video in your filesystem (should be in the same directory) and play it in your video player of choice.

In [16]:
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(Output_video))
Out[16]: